#coding=utf-8
from lxml import etree
from urllib import urlopen
from urlparse import urljoin
import cchardet
import re

def urlget(url):
    '''use urllib to retrieve file and decode,return unicode object'''
    print url
    try:
        data=urlopen(url).read()
        _encoding=cchardet.detect(data)["encoding"]
        #print _encoding
    except:
        print "connection error during get:"+url
        return "connection error during get:"+url
    else:
        return data.decode(_encoding)
        #return data

def import_css(css_url,contents):
    '''merge css imported by the @import url() method'''
    found=re.findall(r'^\s*@import url\([\'"]?([^\'"]*)[\'"]?\)',contents,re.M)
    if not found:
        return contents
    else:
        for x in found:
            if re.search(r'https?://',x,re.I):
                cssfile=x
            else:
                cssfile=urljoin(css_url,x)
            css=urlget(cssfile)
            #escape "\" in css file with lambda(return raw string)
            contents=re.sub(r'.*@import url.*'+x+r'.*',lambda x:css,contents)
        return contents

def merge_css():
    '''merge css to html'''
    linked_css=et.xpath('//link[@rel="stylesheet"][@href]')
    for css in linked_css:
        cssfile=urljoin(url,css.attrib["href"])
        contents=urlget(cssfile)
        css.tag="style"
        etree.strip_attributes(css,css.attrib.keys())
        css.attrib["type"]="text/css"
        #check if there are any other css files imported by @import url() method
        css.text=import_css(cssfile,contents)

def merge_js():
    '''merge javascript to html'''
    linked_js=et.xpath('//script[@type="text/javascript"][@src]')
    for js in linked_js:
        jsfile=urljoin(url,js.attrib["src"])
        contents=urlget(jsfile)
        del js.attrib["src"]
        js.text=contents

def remove_js():
    etree.strip_elements(et,"script")


#url="http://crummy.com/software/BeautifulSoup/bs4/doc/"

url="http://localhost/htmlencoding.htm"
#url="http://www.ruanyifeng.com/blog/2013/08/linux_boot_process.html"
url="http://f543711700.iteye.com/blog/1921430"

data=urlopen(url).read()
enc=cchardet.detect(data)["encoding"]
print enc
_htmlpaser=etree.HTMLParser(encoding=enc)
et=etree.fromstring(data,_htmlpaser)

merge_css()
#merge_js()
remove_js()

#use html <Title> as filename,and remove illegal char
title=et.xpath('/html/*/title')[0]
filename=re.sub(r'[/\:*"<>|\t\r\n]','',title.text).rstrip()+".htm"
output=open(filename,'wb')
output.write(etree.tostring(et,encoding=enc,method="html"))
output.close()

print 'done'